/*
 * Copyright (C) 2016
 *
 * Brick Yang <printfxxx@163.com>
 *
 * This program is free software. You can redistribute it and/or
 * modify it as you like.
 */

/**
 * @file	qman.c
 * @brief	qman driver
 */

#include <linux/init.h>
#include <linux/slab.h>
#include <linux/kernel.h>
#include <linux/module.h>

#include <asm/io.h>

#include <mtrace.h>

#include "qman.h"

#define QM_VALID_BIT	((u32)0x80)

#if defined(CONFIG_ARM64)

/* QCSP CINH register offsets */
#define QCSP_CINH_EQCR_PI	0x3000
#define QCSP_CINH_EQCR_CI	0x3040
#define QCSP_CINH_DQRR_PI	0x3100
#define QCSP_CINH_DQRR_CI	0x3140
#define QCSP_CINH_SDQCR		0x3200
#define QCSP_CINH_MR_PI		0x3300
#define QCSP_CINH_MR_CI		0x3340
#define QCSP_CINH_CFG		0x3500
#define QCSP_CINH_ISR		0x3600
#define QCSP_CINH_IER		0x3640
#define QCSP_CINH_ISDR		0x3680
#define QCSP_CINH_IIR		0x36c0

/* QCSP CENA register offsets */
#define QCSP_CENA_EQCR(n)	(0x0000 + ((u32)(n) << 6))
#define QCSP_CENA_EQCR_IDX(o)	(((u32)(o) - 0x0000) >> 6)
#define QCSP_CENA_DQRR(n)	(0x1000 + ((u32)(n) << 6))
#define QCSP_CENA_DQRR_IDX(o)	(((u32)(o) - 0x1000) >> 6)
#define QCSP_CENA_MR(n)		(0x2000 + ((u32)(n) << 6))
#define QCSP_CENA_MR_IDX(o)	(((u32)(o) - 0x2000) >> 6)
#define QCSP_CENA_EQCR_PI	0x3000
#define QCSP_CENA_EQCR_CI	0x3040
#define QCSP_CENA_DQRR_PI	0x3100
#define QCSP_CENA_DQRR_CI	0x3140
#define QCSP_CENA_MR_PI		0x3300
#define QCSP_CENA_MR_CI		0x3340
#define QCSP_CENA_CR		0x3800
#define QCSP_CENA_RR(n)		(0x3900 + ((u32)(n) << 6))

#elif defined(CONFIG_PPC)

#define QCSP_CINH_EQCR_PI	0x0000
#define QCSP_CINH_EQCR_CI	0x0004
#define QCSP_CINH_DQRR_PI	0x0040
#define QCSP_CINH_DQRR_CI	0x0044
#define QCSP_CINH_SDQCR		0x0054
#define QCSP_CINH_MR_PI		0x0080
#define QCSP_CINH_MR_CI		0x0084
#define QCSP_CINH_CFG		0x0100
#define QCSP_CINH_ISR		0x0e00
#define QCSP_CINH_IER		0x0e04
#define QCSP_CINH_ISDR		0x0e08
#define QCSP_CINH_IIR		0x0e0c

/* QCSP CENA register offsets */
#define QCSP_CENA_EQCR(n)	(0x0000 + ((u32)(n) << 6))
#define QCSP_CENA_EQCR_IDX(o)	(((u32)(o) - 0x0000) >> 6)
#define QCSP_CENA_DQRR(n)	(0x1000 + ((u32)(n) << 6))
#define QCSP_CENA_DQRR_IDX(o)	(((u32)(o) - 0x1000) >> 6)
#define QCSP_CENA_MR(n)		(0x2000 + ((u32)(n) << 6))
#define QCSP_CENA_MR_IDX(o)	(((u32)(o) - 0x2000) >> 6)
#define QCSP_CENA_EQCR_PI	0x3000
#define QCSP_CENA_EQCR_CI	0x3100
#define QCSP_CENA_DQRR_PI	0x3200
#define QCSP_CENA_DQRR_CI	0x3300
#define QCSP_CENA_MR_PI		0x3400
#define QCSP_CENA_MR_CI		0x3500
#define QCSP_CENA_CR		0x3800
#define QCSP_CENA_RR(n)		(0x3900 + ((u32)(n) << 6))

#else
#error "Unsupported platform"
#endif

/* SDQCR attribute codes */
#define QCSP_SDQCR_FC_SHIFT	29
#define QCSP_SDQCR_FC_MASK	0x1
#define QCSP_SDQCR_DP_SHIFT	28
#define QCSP_SDQCR_DP_MASK	0x1
#define QCSP_SDQCR_DCT_SHIFT	24
#define QCSP_SDQCR_DCT_MASK	0x3
#define QCSP_SDQCR_TOK_SHIFT	16
#define QCSP_SDQCR_TOK_MASK	0xff
#define QCSP_SDQCR_SRC_SHIFT	0
#define QCSP_SDQCR_SRC_MASK	0xffff

/* opaque token for static dequeues */
#define QCSP_SDQCR_TOKEN	0xbb

enum qcsp_sdqcr_fc {
	qcsp_sdqcr_fc_one = 0,
	qcsp_sdqcr_fc_up_to_3
};
enum qcsp_sdqcr_dp {
	qcsp_sdqcr_dp_rr = 0,
	qcsp_sdqcr_dp_prio
};
enum qcsp_sdqcr_dct {
	qcsp_sdqcr_dct_null = 0,
	qcsp_sdqcr_dct_prio_ics,
	qcsp_sdqcr_dct_active_ics,
	qcsp_sdqcr_dct_active
};

#if defined(CONFIG_ARM64)
#define dccvac(p)	{ asm volatile("dc cvac, %0;" : : "r" (p) : "memory"); }
#define dcivac(p)	{ asm volatile("dc ivac, %0;" : : "r" (p) : "memory"); }
#elif defined(CONFIG_PPC_E500MC)
#define dccvac(p)	{ asm volatile("dcbf 0,  %0;" : : "r" (p) : "memory"); }
#define dcivac(p)	dccvac(p)
#else
#error "Unsupported platform"
#endif

extern u16 qm_channel_pool1;

/* Portal Access */

static inline u32 qman_cinh_read(struct qman_swp *s, u32 offset)
{
	__be32 be32;

	be32 = __raw_readl(s->addr_cinh + offset);
	return be32_to_cpu(be32);
}

static inline void qman_cinh_write(struct qman_swp *s, u32 offset,
				   u32 val)
{
	__be32 be32;

	be32 = cpu_to_be32(val);
	__raw_writel(be32, s->addr_cinh + offset);
}

static inline void __iomem *qman_cinh_base(struct qman_swp *s, u32 offset)
{
	BUG_ON(!IS_ALIGNED(offset, SMP_CACHE_BYTES));
	return s->addr_cinh + offset;
}

static inline u32 qman_cena_read(struct qman_swp *s, u32 offset)
{
	__be32 be32;

	be32 = __raw_readl(s->addr_cena + offset);
	return be32_to_cpu(be32);
}

static inline void qman_cena_write(struct qman_swp *s, u32 offset,
				   u32 val)
{
	__be32 be32;

	be32 = cpu_to_be32(val);
	__raw_writel(be32, s->addr_cena + offset);
}

static inline void __iomem *qman_cena_base(struct qman_swp *s, u32 offset)
{
	BUG_ON(!IS_ALIGNED(offset, SMP_CACHE_BYTES));
	return s->addr_cena + offset;
}

static inline u8 qm_cyc_diff(u8 mask, u8 left, u8 right)
{
	return (right - left) & mask;
}

#define QCSP_CFG_EST_SHIFT	28
#define QCSP_CFG_EP_SHIFT	26
#define QCSP_CFG_EPM_SHIFT	24
#define QCSP_CFG_DQRR_MF_SHIFT	20
#define QCSP_CFG_DP_SHIFT	18
#define QCSP_CFG_DCM_SHIFT	16
#define QCSP_CFG_SDESTH_SHIFT	12
#define QCSP_CFG_WN_SHIFT	10
#define QCSP_CFG_SD_SHIFT	9
#define QCSP_CFG_MM_SHIFT	8
#define QCSP_CFG_RE_SHIFT	7
#define QCSP_CFG_RP_SHIFT	6
#define QCSP_CFG_SE_SHIFT	5
#define QCSP_CFG_SP_SHIFT	4
#define QCSP_CFG_SDESTL_SHIFT	0

static inline u32 qman_set_qcsp_cfg(u8 est, u8 ep, u8 epm, u8 dqrr_mf,
				    u8 dp, u8 dcm, u8 wn, u8 sd, u8 mm,
				    u8 re, u8 rp, u8 se, u8 sp)
{
	return est << QCSP_CFG_EST_SHIFT |
	       ep << QCSP_CFG_EP_SHIFT |
	       epm << QCSP_CFG_EPM_SHIFT |
	       dqrr_mf << QCSP_CFG_DQRR_MF_SHIFT |
	       dp << QCSP_CFG_DP_SHIFT |
	       dcm << QCSP_CFG_DCM_SHIFT |
	       wn << QCSP_CFG_WN_SHIFT |
	       sd << QCSP_CFG_SD_SHIFT |
	       mm << QCSP_CFG_MM_SHIFT |
	       re << QCSP_CFG_RE_SHIFT |
	       rp << QCSP_CFG_RP_SHIFT |
	       se << QCSP_CFG_SE_SHIFT |
	       sp << QCSP_CFG_SP_SHIFT;
}

struct qman_swp *qman_swp_init(const struct qman_swp_desc *d)
{
	u8 i, n;
	u32 reg;
	bool est, re, eqcr_vb;
	void __iomem *p;
	struct qman_swp *s;

	if (!(s = kmalloc(sizeof(*s), GFP_KERNEL)) || MTRACE_KMEM_ADD(s)) {
		goto err;
	}
	s->desc = d;
	s->sdq = 0;
	s->sdq |= qcsp_sdqcr_fc_up_to_3 << QCSP_SDQCR_FC_SHIFT;
	s->sdq |= qcsp_sdqcr_dp_prio << QCSP_SDQCR_DP_SHIFT;
	s->sdq |= qcsp_sdqcr_dct_prio_ics << QCSP_SDQCR_DCT_SHIFT;
	s->sdq |= QCSP_SDQCR_TOKEN << QCSP_SDQCR_TOK_SHIFT;

	s->dqrr.pi_ci_mask = 0xf;

	s->addr_cena = d->cena_bar;
	s->addr_cinh = d->cinh_bar;

	qman_cinh_write(s, QCSP_CINH_SDQCR, 0);
	qman_cinh_write(s, QCSP_CINH_IER, 0);
	qman_cinh_write(s, QCSP_CINH_ISDR, 0xffffffff);
	qman_cinh_write(s, QCSP_CINH_IIR, 1);
	qman_cinh_write(s, QCSP_CINH_ISR, 0xffffffff);

	est = (s->desc->flags & SWP_F_EST) ? true : false;
	re = (s->desc->flags & SWP_F_RE) ? true : false;
	eqcr_vb = (s->desc->flags & SWP_F_EQCR_VB) ? true : false;
	reg = qman_set_qcsp_cfg(est ? 4 : 0,			/* EQCR_CI stashing threshold */
				est ? 1 : 0,			/* EQCR_CI stashing priority */
				eqcr_vb ? 2 : 1,		/* EQCR production notification mode */
				s->dqrr.pi_ci_mask,		/* DQRR max fill */
				0,				/* DQRR push mode */
				0,				/* DCM: CI write mode */
				0,				/* Writes cacheable */
				0,				/* mem stashing drop enable */
				0,				/* MM: CI write mode */
				re ? 1 : 0,			/* DQRR entry stashing enable */
				re ? 1 : 0,			/* DQRR entry stashing priority */
				0,				/* mem stashing enable */
				0);				/* mem stashing priority */
	qman_cinh_write(s, QCSP_CINH_CFG, reg);

	p = qman_cena_base(s, QCSP_CENA_RR(1));
	dcivac(p);
	if (__raw_readb(p)) {
		s->cr.valid_bit = 0;
	} else {
		s->cr.valid_bit = QM_VALID_BIT;
	}

	reg = qman_cinh_read(s, QCSP_CINH_DQRR_PI);
	s->dqrr.valid_bit = reg & (s->dqrr.pi_ci_mask + 1) ? QM_VALID_BIT : 0;
	s->dqrr.pi = reg & s->dqrr.pi_ci_mask;
	s->dqrr.ci = qman_cinh_read(s, QCSP_CINH_DQRR_CI) & s->dqrr.pi_ci_mask;
	s->dqrr.fi = s->dqrr.ci;
	if (s->dqrr.pi != s->dqrr.ci) {
		pr_err("qman: DQRR ring is not zero, portal is not clean\n");
		goto err;
	}

	s->eqcr.pi_ci_mask = 0x7;
	reg = qman_cinh_read(s, QCSP_CINH_EQCR_PI);
	s->eqcr.valid_bit = reg & (s->eqcr.pi_ci_mask + 1) ? QM_VALID_BIT : 0;
	s->eqcr.pi = reg & s->eqcr.pi_ci_mask;
	s->eqcr.ci = qman_cinh_read(s, QCSP_CINH_EQCR_CI) & s->eqcr.pi_ci_mask;
	s->eqcr.fi = s->eqcr.pi;
	if (s->eqcr.pi != s->eqcr.ci) {
		pr_err("qman: EQCR ring is not zero, portal is not clean\n");
		goto err;
	}

	s->mr.pi_ci_mask = 0x7;
	reg = qman_cinh_read(s, QCSP_CINH_MR_PI);
	s->mr.valid_bit = reg & (s->mr.pi_ci_mask + 1) ? QM_VALID_BIT : 0;
	s->mr.pi = reg & s->mr.pi_ci_mask;
	s->mr.ci = qman_cinh_read(s, QCSP_CINH_MR_CI) & s->mr.pi_ci_mask;
	s->mr.fi = s->mr.pi;
	if (s->mr.pi != s->mr.ci) {
		pr_err("qman: MR ring is not zero, portal is not clean\n");
		goto err;
	}

	n = d->cena_size >> L1_CACHE_SHIFT;
	for (i = 0; i < n; i++) {
		dcivac(d->cena_bar + (i << L1_CACHE_SHIFT));
	}

	return s;
err:
	if (s) {
		MTRACE_KMEM_DEL(s);
		kfree(s);
	}
	return NULL;
}

void qman_swp_finish(struct qman_swp *s)
{
	qman_cinh_write(s, QCSP_CINH_SDQCR, 0);
	qman_cinh_write(s, QCSP_CINH_IER, 0);
	qman_cinh_write(s, QCSP_CINH_ISDR, 0xffffffff);
	qman_cinh_write(s, QCSP_CINH_IIR, 1);
	qman_cinh_write(s, QCSP_CINH_ISR, 0xffffffff);

	MTRACE_KMEM_DEL(s);
	kfree(s);
}

void __iomem *qman_swp_cr_next(struct qman_swp *s, u8 *cmd_verb)
{
	void __iomem *p;

	p = qman_cena_base(s, QCSP_CENA_CR);
	*cmd_verb = s->cr.valid_bit;

	return p;
}

const void __iomem *qman_swp_rr_next(struct qman_swp *s, u8 cmd_verb)
{
	const void __iomem *p;

	p = qman_cena_base(s, QCSP_CENA_RR(s->cr.valid_bit ? 1 : 0));
	if (__raw_readb(p) != cmd_verb) {
		dma_rmb();
		dcivac(p);
		prefetch(p);
		return NULL;
	}

	s->cr.valid_bit ^= QM_VALID_BIT;
	return p;
}

void __iomem *qman_swp_eqcr_next(struct qman_swp *s, u8 *cmd_verb)
{
	u8 free;

	free = s->eqcr.pi_ci_mask - qm_cyc_diff(s->eqcr.pi_ci_mask, s->eqcr.ci, s->eqcr.pi);
	if (!free) {
		if (s->desc->flags & SWP_F_EST) {
			s->eqcr.ci = qman_cena_read(s, QCSP_CENA_EQCR_CI) & s->eqcr.pi_ci_mask;
		} else {
			s->eqcr.ci = qman_cinh_read(s, QCSP_CINH_EQCR_CI) & s->eqcr.pi_ci_mask;
		}
		free = s->eqcr.pi_ci_mask - qm_cyc_diff(s->eqcr.pi_ci_mask, s->eqcr.ci, s->eqcr.pi);
	}

	if (free) {
		*cmd_verb = s->eqcr.valid_bit;
		return qman_cena_base(s, QCSP_CENA_EQCR(s->eqcr.pi & s->eqcr.pi_ci_mask));
	} else {
		return NULL;
	}
}

const void __iomem *qman_swp_dqrr_next(struct qman_swp *s)
{
	u8 n, avail;
	void __iomem *p;
	unsigned long flags;

	flags = s->desc->flags;
	avail = qm_cyc_diff(s->dqrr.pi_ci_mask, s->dqrr.ci, s->dqrr.pi);
	if (!avail) {
		if (flags & SWP_F_DQRR_VB) {
			p = qman_cena_base(s, QCSP_CENA_DQRR(s->dqrr.pi & s->dqrr.pi_ci_mask));
			if ((__raw_readb(p) & QM_VALID_BIT) == s->dqrr.valid_bit) {
				s->dqrr.pi = (s->dqrr.pi + 1) & s->dqrr.pi_ci_mask;
				if (!s->dqrr.pi) {
					s->dqrr.valid_bit ^= QM_VALID_BIT;
				}
			}
		} else {
			s->dqrr.pi = qman_cinh_read(s, QCSP_CINH_DQRR_PI) & s->dqrr.pi_ci_mask;
		}
		avail = qm_cyc_diff(s->dqrr.pi_ci_mask, s->dqrr.ci, s->dqrr.pi);
	}

	if (!(flags & SWP_F_RE) && (flags & SWP_F_DQRR_VB)) {
		n = qm_cyc_diff(s->dqrr.pi_ci_mask, s->dqrr.fi, s->dqrr.pi);
		if (n < s->dqrr.pi_ci_mask) {
			p = qman_cena_base(s, QCSP_CENA_DQRR(s->dqrr.pi & s->dqrr.pi_ci_mask));
			dma_rmb();
			dcivac(p);
			prefetch(p);
		}
	}

	if (!avail) {
		return NULL;
	}

	p = qman_cena_base(s, QCSP_CENA_DQRR(s->dqrr.ci & s->dqrr.pi_ci_mask));
	if (!(flags & SWP_F_DQRR_VB)) {
		dma_rmb();
		dcivac(p);
		prefetch(p);
	}

	return p;
}

const void __iomem *qman_swp_mr_next(struct qman_swp *s)
{
	u8 n, avail;
	void __iomem *p;
	unsigned long flags;

	flags = s->desc->flags;
	avail = qm_cyc_diff(s->mr.pi_ci_mask, s->mr.ci, s->mr.pi);
	if (!avail) {
		if (flags & SWP_F_MR_VB) {
			p = qman_cena_base(s, QCSP_CENA_MR(s->mr.ci & s->mr.pi_ci_mask));
			if ((__raw_readb(p) & QM_VALID_BIT) == s->mr.valid_bit) {
				s->mr.pi = (s->mr.pi + 1) & s->mr.pi_ci_mask;
				if (!s->mr.pi) {
					s->mr.valid_bit ^= QM_VALID_BIT;
				}
			}
		} else {
			s->mr.pi = qman_cinh_read(s, QCSP_CINH_MR_PI) & s->mr.pi_ci_mask;
		}
		avail = qm_cyc_diff(s->mr.pi_ci_mask, s->mr.ci, s->mr.pi);
	}

	if (flags & SWP_F_MR_VB) {
		n = qm_cyc_diff(s->mr.pi_ci_mask, s->mr.fi, s->mr.pi);
		if (n < s->mr.pi_ci_mask) {
			p = qman_cena_base(s, QCSP_CENA_MR(s->mr.pi & s->mr.pi_ci_mask));
			dma_rmb();
			dcivac(p);
			prefetch(p);
		}
	}

	if (!avail) {
		return NULL;
	}

	p = qman_cena_base(s, QCSP_CENA_MR(s->mr.ci & s->mr.pi_ci_mask));
	if (!(flags & SWP_F_MR_VB)) {
		dma_rmb();
		dcivac(p);
		prefetch(p);
	}

	return p;
}

void qman_swp_cr_submit(struct qman_swp *s, void __iomem *p, u8 cmd_verb)
{
	dma_wmb();
	__raw_writeb(cmd_verb, p);
	dccvac(p);
	p = qman_cena_base(s, QCSP_CENA_RR(s->cr.valid_bit ? 1 : 0));
	dcivac(p);
	prefetch(p);
}

int qman_swp_eqcr_submit(struct qman_swp *s, void __iomem *p, u8 cmd_verb, bool flush)
{
	u8 i, n;
	unsigned long flags;

	flags = s->desc->flags;

	if (p) {
		i = QCSP_CENA_EQCR_IDX(p - qman_cena_base(s, 0));
		if (i != s->eqcr.pi) {
			return -EINVAL;
		}
		if (flags & SWP_F_EQCR_VB) {
			dma_wmb();
		}
		__raw_writeb(cmd_verb, p);
		dccvac(p);
		s->eqcr.pi = (s->eqcr.pi + 1) & s->eqcr.pi_ci_mask;
		if ((flags & SWP_F_EQCR_VB) && !s->eqcr.pi) {
			s->eqcr.valid_bit ^= QM_VALID_BIT;
		}
	}

	if (!(flags & SWP_F_EQCR_VB)) {
		n = qm_cyc_diff(s->eqcr.pi_ci_mask, s->eqcr.fi, s->eqcr.pi);
		if (n && ((n >= s->eqcr.pi_ci_mask >> 2) || flush)) {
			dma_wmb();
			qman_cena_write(s, QCSP_CENA_EQCR_PI, s->eqcr.pi);
			dccvac(qman_cena_base(s, QCSP_CENA_EQCR_PI));
			s->eqcr.fi = s->eqcr.pi;
		}
	}

	return 0;
}

int qman_swp_dqrr_consume(struct qman_swp *s, const void __iomem *p, bool flush)
{
	u8 i, n;

	if (p) {
		i = QCSP_CENA_DQRR_IDX(p - qman_cena_base(s, 0));
		if (i != s->dqrr.ci) {
			return -EINVAL;
		}
		s->dqrr.ci = (s->dqrr.ci + 1) & s->dqrr.pi_ci_mask;
	}

	n = qm_cyc_diff(s->dqrr.pi_ci_mask, s->dqrr.fi, s->dqrr.ci);
	if (n && ((n >= s->dqrr.pi_ci_mask >> 2) || flush)) {
		qman_cinh_write(s, QCSP_CINH_DQRR_CI, s->dqrr.ci);
		s->dqrr.fi = s->dqrr.ci;
	}

	return 0;
}

int qman_swp_mr_consume(struct qman_swp *s, const void __iomem *p, bool flush)
{
	u8 i, n;

	if (p) {
		i = QCSP_CENA_MR_IDX(p - qman_cena_base(s, 0));
		if (i != s->mr.ci) {
			return -EINVAL;
		}
		s->mr.ci = (s->mr.ci + 1) & s->mr.pi_ci_mask;
	}

	n = qm_cyc_diff(s->mr.pi_ci_mask, s->mr.fi, s->mr.ci);
	if (n && ((n >= s->mr.pi_ci_mask >> 2) || flush)) {
		qman_cinh_write(s, QCSP_CINH_MR_CI, s->mr.ci);
		s->mr.fi = s->mr.ci;
	}

	return 0;
}

u8 qman_swp_push_chid_to_idx(u16 chid)
{
	return chid - qm_channel_pool1 + 1;
}

void qman_swp_push_get(struct qman_swp *s, u8 idx, bool *en)
{
	u16 src = (s->sdq >> QCSP_SDQCR_SRC_SHIFT) & QCSP_SDQCR_SRC_MASK;

	WARN_ON(idx > 15);
	*en = src & (0x8000 >> idx) ? true : false;
}

void qman_swp_push_set(struct qman_swp *s, u8 idx, bool en)
{
	u16 src;

	WARN_ON(idx > 15);
	if (en) {
		s->sdq |= 0x8000 >> idx;
	} else {
		s->sdq &= ~(0x8000 >> idx);
	}

	src = (s->sdq >> QCSP_SDQCR_SRC_SHIFT) & QCSP_SDQCR_SRC_MASK;
	if (src) {
		qman_cinh_write(s, QCSP_CINH_SDQCR, s->sdq);
	} else {
		qman_cinh_write(s, QCSP_CINH_SDQCR, 0);
	}
}
